{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def deepnn(x):\n",
    "  with tf.name_scope('reshape'):\n",
    "    x_image = tf.reshape(x, [-1, 28, 28, 1])\n",
    "\n",
    "  # First convolutional layer - maps one grayscale image to 32 feature maps.\n",
    "  with tf.name_scope('conv1'):\n",
    "    W_conv1 = weight_variable([5, 5, 1, 32])\n",
    "    b_conv1 = bias_variable([32])\n",
    "    h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)\n",
    "\n",
    "  # Pooling layer - downsamples by 2X.\n",
    "  with tf.name_scope('pool1'):\n",
    "    h_pool1 = max_pool_2x2(h_conv1)\n",
    "\n",
    "  # Second convolutional layer -- maps 32 feature maps to 64.\n",
    "  with tf.name_scope('conv2'):\n",
    "    W_conv2 = weight_variable([5, 5, 32, 64])\n",
    "    b_conv2 = bias_variable([64])\n",
    "    h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)\n",
    "\n",
    "  # Second pooling layer.\n",
    "  with tf.name_scope('pool2'):\n",
    "    h_pool2 = max_pool_2x2(h_conv2)\n",
    "\n",
    "  # Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image\n",
    "  # is down to 7x7x64 feature maps -- maps this to 1024 features.\n",
    "  with tf.name_scope('fc1'):\n",
    "    W_fc1 = weight_variable([7 * 7 * 64, 1024])\n",
    "    b_fc1 = bias_variable([1024])\n",
    "\n",
    "    h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64])\n",
    "    h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)\n",
    "\n",
    "  # Dropout - controls the complexity of the model, prevents co-adaptation of\n",
    "  # features.\n",
    "  with tf.name_scope('dropout'):\n",
    "    keep_prob = tf.placeholder(tf.float32)\n",
    "    h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)\n",
    "\n",
    "  # Map the 1024 features to 10 classes, one for each digit\n",
    "  with tf.name_scope('fc2'):\n",
    "    W_fc2 = weight_variable([1024, 10])\n",
    "    b_fc2 = bias_variable([10])\n",
    "\n",
    "    y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2\n",
    "  return y_conv, keep_prob\n",
    "\n",
    "\n",
    "def conv2d(x, W):\n",
    "  \"\"\"conv2d returns a 2d convolution layer with full stride.\"\"\"\n",
    "  return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')\n",
    "\n",
    "\n",
    "def max_pool_2x2(x):\n",
    "  \"\"\"max_pool_2x2 downsamples a feature map by 2X.\"\"\"\n",
    "  return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],\n",
    "                        strides=[1, 2, 2, 1], padding='SAME')\n",
    "\n",
    "\n",
    "def weight_variable(shape):\n",
    "  \"\"\"weight_variable generates a weight variable of a given shape.\"\"\"\n",
    "  initial = tf.truncated_normal(shape, stddev=0.1)\n",
    "  return tf.Variable(initial)\n",
    "\n",
    "\n",
    "def bias_variable(shape):\n",
    "  \"\"\"bias_variable generates a bias variable of a given shape.\"\"\"\n",
    "  initial = tf.constant(0.1, shape=shape)\n",
    "  return tf.Variable(initial)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def main():\n",
    "  # Import data\n",
    "  mnist = input_data.read_data_sets(\"../datasets/MNIST_data\", one_hot=True)\n",
    "  # Create the model\n",
    "  x = tf.placeholder(tf.float32, [None, 784])\n",
    "\n",
    "  # Define loss and optimizer\n",
    "  y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "  # Build the graph for the deep net\n",
    "  y_conv, keep_prob = deepnn(x)\n",
    "\n",
    "  with tf.name_scope('loss'):\n",
    "    cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_,\n",
    "                                                            logits=y_conv)\n",
    "  cross_entropy = tf.reduce_mean(cross_entropy)\n",
    "\n",
    "  with tf.name_scope('adam_optimizer'):\n",
    "    train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n",
    "\n",
    "  with tf.name_scope('accuracy'):\n",
    "    correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))\n",
    "    correct_prediction = tf.cast(correct_prediction, tf.float32)\n",
    "  accuracy = tf.reduce_mean(correct_prediction)\n",
    "\n",
    "  train_writer = tf.summary.FileWriter(\"model\")\n",
    "  train_writer.add_graph(tf.get_default_graph())\n",
    "\n",
    "  with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    for i in range(1000):\n",
    "      batch = mnist.train.next_batch(50)\n",
    "      if i % 100 == 0:\n",
    "        train_accuracy = accuracy.eval(feed_dict={\n",
    "            x: batch[0], y_: batch[1], keep_prob: 1.0})\n",
    "        print('step %d, training accuracy %g' % (i, train_accuracy))\n",
    "      train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})\n",
    "\n",
    "    print('test accuracy %g' % accuracy.eval(feed_dict={\n",
    "        x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ../datasets/MNIST_data/train-images-idx3-ubyte.gz\n",
      "Extracting ../datasets/MNIST_data/train-labels-idx1-ubyte.gz\n",
      "Extracting ../datasets/MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting ../datasets/MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "step 0, training accuracy 0.18\n",
      "step 100, training accuracy 0.76\n",
      "step 200, training accuracy 0.92\n",
      "step 300, training accuracy 1\n",
      "step 400, training accuracy 0.94\n",
      "step 500, training accuracy 0.94\n",
      "step 600, training accuracy 0.96\n",
      "step 700, training accuracy 0.94\n",
      "step 800, training accuracy 0.96\n",
      "step 900, training accuracy 0.96\n",
      "test accuracy 0.962\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "  main()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
